# -*- coding: utf-8 -*-

# @Author: Chen Hua
# @Date  : 2020-07-11
# @Desc  : 中国新闻联播数据

import sys
import datetime
import time

import decimal
from sqlalchemy.sql.elements import and_

from db.MySqlConn import DBSession, JobConfig, engine as eg, Xwlb, IndexValue
import service.job_config_service as js
from util import DateUtil
import tushare as ts
from conf.settings import TUSHARE_TOKEN

##tushare sdk
ts.set_token(TUSHARE_TOKEN)
pro = ts.pro_api()
##链接数据库
session = DBSession()
job_name = 'stock_xwlb_news'

"""
详情：
https://tushare.pro/document/2?doc_id=154
"""

def process_xwlb(dt):
    for _ in range(1):  # 重试三次
        new_xwlb_list = []
        try:
            df = pro.cctv_news(date=DateUtil.date_to_yyyyMMhh_str(dt))
            if df.empty:
                print("没有数据返回，请检查是否出错...", DateUtil.date_to_yyyy_MM_hh_str(dt))
            for index, row in df.iterrows():
                print(row)
                new_xwlb_list.append( Xwlb(dt=dt, title=row['title'], content=row['content']) )
        except Exception as e:
            print(str(e))
            time.sleep(2)
        else:
            return new_xwlb_list


def stock_xwlb_news_info_sync():
    try:
        # df = pro.cctv_news(date='20200806')
        # if df.empty:
        #     print("没有数据返回，请检查是否出错...")
        # for index, row in df.iterrows():
        #     print(row)
        now = datetime.datetime.now()
        begin_dt = datetime.datetime(2020, 8, 1)  ##默认初始从2004-01-01 开始
        ##从数据库获取上次已获取的数据日期
        spider_config = session.query(JobConfig).filter(JobConfig.job_name == job_name).one()
        if spider_config and spider_config.job_mode == 'inc' and spider_config.last_data_at:
            begin_dt = spider_config.last_data_at
            begin_dt = datetime.datetime(begin_dt.year, begin_dt.month, begin_dt.day)

        while begin_dt < now:
            ii_list = process_xwlb(begin_dt)
            print("成功爬取新闻联播 ",  begin_dt.date(), "的信息...")
            for ii in ii_list:
                session.merge(ii)
                session.commit()
            js.update_job_config(job_name, begin_dt)
            begin_dt = begin_dt + datetime.timedelta(days=1)
            time.sleep(31)  # 睡眠1秒
        print('Stock xwlb 信息已保存到表 stock_xwlb_news ...')
    except Exception as e:
        print('获取Stock 新闻联播信息失败.')
        print(str(e))
        sys.exit(1)
    finally:
        session.close()

if __name__ == "__main__":
    stock_xwlb_news_info_sync()
